# -*- coding: utf-8 -*-
# @Time    : 2019/12/11 10:32
# @Author  : ZSQ
# @Email   : zsq199170918@163.com
# @FileName: xuzhou.py
# @Software: PyCharm
import scrapy

import re
import json
import random
import base64
import logging
from functools import wraps
from inspect import getcallargs
from copy import copy, deepcopy
from FDC_spider.settings import USER_AGENTS
from FDC_spider.utils.img_ocr import ImageOcr
from FDC_spider.utils.transfer_time import transfer_date
from FDC_spider.items import FdcEstateGuidItem, FdcBuildingItem, FdcRoomItem

logger = logging.getLogger(__name__)


def deal_exception(func):
    @wraps(func)
    def wrapper(*args, **kwargs):
        try:
            _params = getcallargs(func, *args, **kwargs)
            _ret = int(_params.get('stamp'))
        except Exception as e:
            logger.info('The type of stamp is wrong!  error_msg:{}'.format(e))
        else:
            return func(*args, **kwargs)

    return wrapper


transfer_date = deal_exception(transfer_date)


class XuzhouSpider(scrapy.Spider):
    name = 'xuzhou'
    allowed_domains = ['xzhouse.com.cn']
    start_urls = ['https://www.xzhouse.com.cn/house/item/getItems.do']  # POST
    prj_url = 'https://www.xzhouse.com.cn/house/item/getItemById.do'  # POST   项目详情url
    company_url = 'https://www.xzhouse.com.cn/house/item/getCorpNameByItemId.do'  # POST   开发商url
    permit_li_url = 'https://www.xzhouse.com.cn/house/zxm/getZxmYsxxByItemid.do'  # POST   许可证列表url
    building_li_url = 'https://www.xzhouse.com.cn/house/buildingInfo/getBuildingInfoByItemid.do'  # POST  楼栋列表url
    building_url = 'https://www.xzhouse.com.cn/house/buildingInfo/getZxmBuildingInfoBybId.do'  # POST  楼栋url
    room_li_url = 'https://www.xzhouse.com.cn/house/houseInfo/getHouseInfoImgByCondition.do'  # POST  房号列表url

    def start_requests(self):
        agent = random.choice(USER_AGENTS)
        headers = {
            'User-Agent': agent,
            'Referer': 'https://www.xzhouse.com.cn/html/newHouse/xzhItems.html',
        }
        yield scrapy.Request(
            self.start_urls[0],
            headers=headers,
            method='POST',
        )

    def parse(self, response):
        try:
            resp_dict = json.loads(response.body.decode())
            prj_li = resp_dict.get('obj', [])
            assert resp_dict.get('success', None) and prj_li
        except Exception as e:
            logger.error('获取楼盘列表失败  error_msg:{}'.format(e))
        else:
            for index, prj_dict in enumerate(prj_li):
                item_eg = FdcEstateGuidItem()
                item_eg['projectName'] = prj_dict.get('xmmc', None)
                item_eg['projectId'] = prj_dict.get('id', None)
                item_eg['publicName'] = prj_dict.get('tgmc', None)
                try:
                    assert item_eg['projectId'] and item_eg['projectName']
                except Exception as e:
                    logger.error(
                        '获取 {}-{}-{} 的楼盘名称或id出错  error_msg:{}'.format(index, item_eg['projectName'],
                                                                      item_eg['projectId'], e))
                    continue
                else:
                    # 构造POST请求参数，获取项目详情
                    data = dict(id=item_eg['projectId'])
                    headers = {
                        'Referer': 'https://www.xzhouse.com.cn/html/item/xzhItemInfo.html?itemid={}'.format(
                            item_eg['projectId']),
                    }
                    yield scrapy.FormRequest(
                        self.prj_url,
                        headers=headers,
                        callback=self.parse_project_detail,
                        formdata=data,
                        meta=dict(item_eg=deepcopy(item_eg)),
                    )

    def parse_project_detail(self, response):
        """
        获取项目详情
        :param response:
        :return:
        """
        item_eg = copy(response.meta['item_eg'])
        try:
            resp_dict = json.loads(response.body.decode())
            prj_dict = resp_dict.get('obj', {})
            assert resp_dict.get('success', None) and prj_dict
        except Exception as e:
            logger.error('获取 {}-{} 项目详情失败  error_msg:{}'.format(item_eg['projectName'], item_eg['projectId'], e))
        else:
            item_eg['projectAddress'] = self._get_item(prj_dict, 'xmdz')
            item_eg['saleAddress'] = self._get_item(prj_dict, 'xsdz')
            item_eg['greeningRate'] = self._get_item(prj_dict, 'lhl')
            item_eg['floorAreaRatio'] = self._get_item(prj_dict, 'rjl')
            item_eg['propertyCompany'] = self._get_item(prj_dict, 'wyglgs')
            item_eg['propertyManagementFee'] = self._get_item(prj_dict, 'wyf')
            item_eg['parkingSpacesNum'] = self._get_item(prj_dict, 'tcw')
            item_eg['parkingPrice'] = self._get_item(prj_dict, 'tcwsj')
            item_eg['parkingRental'] = self._get_item(prj_dict, 'tcwzj')
            item_eg['lngBd'], item_eg['latBd'] = self._get_lng_and_lat(self._get_item(prj_dict, 'zb'))
            item_eg['salesTel'] = self._get_item(prj_dict, 'zxdh')
            item_eg['projectBrief'] = self._get_item(prj_dict, 'ms')
            item_eg['handoverDate'] = self._transfer_date(prj_dict, 'sfsj')
            item_eg['commencementDate'] = self._transfer_date(prj_dict, 'kgsj')
            item_eg['completionDate'] = self._transfer_date(prj_dict, 'jgsj')
            item_eg['avgPrice'] = self._get_value(prj_dict, 'itemTj', 'nxsjg')
            item_eg['decorateSituation'] = self._get_value(prj_dict, 'itemZxzk', 'zxzkContent')
            item_eg['buildingCategory'] = self._get_value(prj_dict, 'itemJzlb', 'wylbContent')
            item_eg['buildingForm'] = self._get_value(prj_dict, 'itemJzxs', 'jzxsContent')
            item_eg['totalRoomNum'] = self._get_value(prj_dict, 'itemTj', 'rwts')
            item_eg['totalSoldRoomNum'] = self._get_value(prj_dict, 'itemTj', 'xsts')
            item_eg['totalUnsoldRoomNum'] = self._get_value(prj_dict, 'itemTj', 'ksts')
            item_eg['totalUnSaleableRoomNum'] = self._count_num(item_eg['totalRoomNum'], item_eg['totalSoldRoomNum'],
                                                                item_eg['totalUnsoldRoomNum'])
            item_eg['totalArea'] = self._get_item(prj_dict, 'jzmj')
            item_eg['totalSoldArea'] = self._get_value(prj_dict, 'itemTj', 'xsmj')
            item_eg['totalUnSoldArea'] = self._get_value(prj_dict, 'itemTj', 'ksmj')
            # 构造POST请求参数，获取开发商
            data = dict(itemid=item_eg['projectId'])
            headers = {
                'Referer': 'https://www.xzhouse.com.cn/html/item/xzhItemInfo.html?itemid={}'.format(
                    item_eg['projectId']),
            }
            yield scrapy.FormRequest(
                self.company_url,
                headers=headers,
                callback=self.parse_developer,
                meta=dict(item_eg=deepcopy(item_eg)),
                formdata=data,
            )

    def parse_developer(self, response):
        """
        获取开发商
        :param response:
        :return:
        """
        item_eg = copy(response.meta['item_eg'])
        try:
            resp_dict = json.loads(response.body.decode())
            assert resp_dict.get('success', None)
        except Exception as e:
            logger.error('获取 {}-{} 开发商信息失败  error_msg:{}'.format(item_eg['projectName'], item_eg['projectId'], e))
            item_eg['developerName'] = None
        else:
            item_eg['developerName'] = resp_dict.get('obj', None)
        finally:
            # 构造POST请求参数，获取预售许可证列表
            data = dict(itemid=item_eg['projectId'])
            headers = {
                'Referer': 'https://www.xzhouse.com.cn/html/item/xzhItemInfo.html?itemid={}'.format(
                    item_eg['projectId']),
            }
            yield scrapy.FormRequest(
                self.permit_li_url,
                headers=headers,
                callback=self.parse_permit_li,
                meta=dict(item_eg=deepcopy(item_eg)),
                formdata=data,
            )

    def parse_permit_li(self, response):
        """
        获取预售许可证列表
        :param response:
        :return:
        """
        item_eg = copy(response.meta['item_eg'])
        try:
            resp_li = json.loads(response.body.decode())
            assert resp_li and isinstance(resp_li, list)
        except Exception as e:
            logger.error('获取 {}-{} 许可证列表获取错误  error_msg:{}'.format(item_eg['projectName'], item_eg['projectId'], e))
        else:
            for permit_dict in resp_li:
                item_eg['districtName'] = self._get_item(permit_dict, 'district')
                item_eg['preSalePermit'] = self._get_item(permit_dict, 'certificateNO')
                item_eg['permitId'] = self._get_item(permit_dict, 'saleItemID')
                item_eg['certDate'] = self._get_item(permit_dict, 'certificateTime')
                item_eg['preSaleBuilding'] = self._get_item(permit_dict, 'saleScope')
                item_eg['planningUse'] = self._get_item(permit_dict, 'planUse')
                item_eg['permitTotalRoomNum'] = self._get_item(permit_dict, 'rwzts')
                item_eg['permitUnSoldRoomNum'] = self._get_item(permit_dict, 'kxsts')
                item_eg['permitSoldRoomNum'] = self._get_item(permit_dict, 'cjts')
                item_eg['permitUnSaleableRoomNum'] = self._count_num(item_eg['permitTotalRoomNum'],
                                                                     item_eg['permitUnSoldRoomNum'],
                                                                     item_eg['permitSoldRoomNum'])
                yield item_eg
            # 构造POST请求参数，获取楼栋列表
            data = dict(itemid=item_eg['projectId'])
            headers = {
                'Referer': 'https://www.xzhouse.com.cn/html/item/xzhSaleInfo.html?id={}'.format(item_eg['projectId']),
            }
            yield scrapy.FormRequest(
                self.building_li_url,
                headers=headers,
                callback=self.parse_building_li,
                meta=dict(item_eg=deepcopy(item_eg)),
                formdata=data,
            )

    def parse_building_li(self, response):
        """
        获取楼栋列表
        :param response:
        :return:
        """
        item_eg = copy(response.meta['item_eg'])
        try:
            resp_dict = json.loads(response.body.decode())
            bd_li = resp_dict.get('obj', [])
            assert resp_dict.get('success', None) and bd_li and isinstance(bd_li, list)
        except Exception as e:
            logger.error('获取 {}-{} 楼栋列表失败  error_msg:{}'.format(item_eg['projectName'], item_eg['projectId'], e))
        else:
            headers = {
                'Referer': 'https://www.xzhouse.com.cn/html/item/xzhSaleInfo.html?id={}'.format(item_eg['projectId']),
            }
            for bd_dict in bd_li:
                item_bd = FdcBuildingItem()
                item_bd['projectName'] = item_eg['projectName']
                item_bd['projectId'] = item_eg['projectId']
                item_bd['blockName'] = self._get_item(bd_dict, 'BuildingName')
                item_bd['buildingId'] = self._get_item(bd_dict, 'BuildingInfo_ID')
                item_bd['preSalePermit'] = self._get_item(bd_dict, 'CertificateNO')
                # 构造POST请求参数，获取楼栋信息
                data_bd = dict(bId=item_bd['buildingId'])
                yield scrapy.FormRequest(
                    self.building_url,
                    headers=headers,
                    callback=self.parse_building_detail,
                    meta=dict(item_bd=deepcopy(item_bd)),
                    formdata=data_bd,
                )
                # 构造POST请求参数，获取房号列表
                data_rm = dict(
                    pageSize='10',
                    currPageNo='1',
                    lx='000',
                    buildingId=item_bd['buildingId'],
                    unitNo='',
                    houseId='',
                    saleHouseUse='',
                )
                yield scrapy.FormRequest(
                    self.room_li_url,
                    headers=headers,
                    callback=self.parse_room_li,
                    meta=dict(item_bd=deepcopy(item_bd), page_num=1),
                    formdata=data_rm,
                )

    def parse_building_detail(self, response):
        """
        获取楼栋信息
        :param response:
        :return:
        """
        item_bd = copy(response.meta['item_bd'])
        try:
            resp_dict = json.loads(response.body.decode())
            sub_bd_li = resp_dict.get('obj', [])
            bd_dict = sub_bd_li[0]
            assert resp_dict.get('success', None) and bd_dict and isinstance(bd_dict, dict)
        except Exception as e:
            logger.error(
                '获取 {}({}) - {} - {}({}) 楼栋信息失败  error_msg:{}'.format(item_bd['projectName'], item_bd['projectId'],
                                                                      item_bd['preSalePermit'], item_bd['blockName'],
                                                                      item_bd['buildingId'], e))
        else:
            item_bd['roomTotalNo'] = self._get_item(bd_dict, 'rwzts')
            item_bd['soldRoomNo'] = self._get_item(bd_dict, 'cjts')
            item_bd['unsoldRoomNo'] = self._get_item(bd_dict, 'kxsts')
            item_bd['unsaleableRoomNo'] = self._count_num(item_bd['roomTotalNo'], item_bd['soldRoomNo'],
                                                          item_bd['unsoldRoomNo'])
            item_bd['buildingTotalArea'] = self._get_item(bd_dict, 'zmj')
            item_bd['buildingSoldArea'] = self._get_item(bd_dict, 'cjmj')
            item_bd['buildingUnsoldArea'] = self._get_item(bd_dict, 'kxsmj')
            yield item_bd

    def parse_room_li(self, response):
        """
        获取房号列表
        :param response:
        :return:
        """
        item_bd = copy(response.meta['item_bd'])
        page_num = copy(response.meta['page_num'])
        try:
            resp_dict = json.loads(response.body.decode())
            rm_li = resp_dict.get('obj', [])
            assert resp_dict.get('success', None) and rm_li and isinstance(rm_li, list)
            total_page_num = int(resp_dict['attributes']['pageCount'])
        except Exception as e:
            logger.error(
                '{}({}) - {} - {}({}) - 第{}页  房号列表获取失败  error_msg:{}'.format(item_bd['projectName'],
                                                                             item_bd['projectId'],
                                                                             item_bd['preSalePermit'],
                                                                             item_bd['blockName'],
                                                                             item_bd['buildingId'], page_num, e))
        else:
            imo = ImageOcr()
            for rm_dict in rm_li:
                item_rm = FdcRoomItem()
                item_rm['projectName'] = item_bd['projectName']
                item_rm['projectId'] = item_bd['projectId']
                item_rm['preSalePermit'] = item_bd['preSalePermit']
                item_rm['blockName'] = item_bd['blockName']
                item_rm['buildingId'] = item_bd['buildingId']
                img_b64_str = self._get_item(rm_dict, 'base64String')
                row_num = self._get_item(rm_dict, 'rownumber')
                try:
                    img_b64 = base64.b64decode(img_b64_str)
                    imo.bstr = img_b64
                    code = imo.main(_lang='chi_sim')
                    room_values_li = self._deal_room_str(code)
                    assert len(room_values_li) == 10
                except Exception as e:
                    logger.error(
                        '{}({}) - {} - {}({}) - 第{}页 - {}  房号图片识别失败  error_msg:{}'.format(item_bd['projectName'],
                                                                                          item_bd['projectId'],
                                                                                          item_bd['preSalePermit'],
                                                                                          item_bd['blockName'],
                                                                                          item_bd['buildingId'],
                                                                                          page_num, row_num,
                                                                                          e))
                else:
                    item_rm['saleStatus'] = self._format_str(room_values_li[0])
                    item_rm['unitNo'] = self._format_str(room_values_li[1])
                    item_rm['roomNo'] = self._format_str(room_values_li[2])
                    item_rm['roomUse'] = self._format_str(room_values_li[3])
                    item_rm['roomStructure'] = self._format_str(room_values_li[4])
                    item_rm['roomNature'] = self._format_str(room_values_li[5])
                    item_rm['roomArea'] = self._format_str(room_values_li[6])
                    item_rm['innerArea'] = self._format_str(room_values_li[7])
                    item_rm['apportionmentArea'] = self._format_str(room_values_li[8])
                    item_rm['unitPrice'] = self._format_str(room_values_li[9])
                    yield item_rm
            del imo

            # 翻页
            if int(page_num) < int(total_page_num):
                page_num += 1
                # 构造POST请求参数，获取下一页房号列表
                headers = {
                    'Referer': 'https://www.xzhouse.com.cn/html/item/xzhSaleInfo.html?id={}'.format(
                        item_bd['projectId']),
                }
                data = dict(
                    pageSize='10',
                    currPageNo=str(page_num),
                    lx='000',
                    buildingId=item_bd['buildingId'],
                    unitNo='',
                    houseId='',
                    saleHouseUse='',
                )
                yield scrapy.FormRequest(
                    self.room_li_url,
                    headers=headers,
                    callback=self.parse_room_li,
                    meta=dict(item_bd=deepcopy(item_bd), page_num=page_num),
                    formdata=data,
                )

    @staticmethod
    def _get_item(_dict, _str):
        if _dict and _str:
            return str(_dict.get(_str, None)).strip() if _dict.get(_str, None) else None

    @staticmethod
    def _get_lng_and_lat(_str):
        if _str and ',' in _str:
            coordinate_li = _str.split(',')
            if len(coordinate_li) == 2 and coordinate_li[0] and coordinate_li[1]:
                return coordinate_li[0], coordinate_li[1]
        return None, None

    def _transfer_date(self, _dict, _str):
        return transfer_date(self._get_item(_dict, _str))

    def _get_value(self, _dict, _str, _key):
        try:
            _ret = eval(self._get_item(_dict, _str))
            assert isinstance(_ret, list) or isinstance(_ret, dict)
        except:
            return
        else:
            if isinstance(_ret, list):
                _value_li = list()
                for _value_dict in _ret:
                    if _value_dict.get(_key, None):
                        _value_li.append(_value_dict.get(_key, None))
                return _value_li if _value_li else None
            return _ret.get(_key, None)

    @staticmethod
    def _count_num(_num1, _num2, _num3):
        try:
            _ret = int(_num1) - int(_num2) - int(_num3)
        except:
            return
        else:
            return _ret

    @staticmethod
    def _deal_room_str(_str):
        regex_1 = re.compile(r'\s{4,}')
        regex_2 = re.compile(r'\s')
        if _str:
            ret = regex_2.sub('', regex_1.sub(',', _str))
            ret_li = ret.split(',')
            return ret_li

    @staticmethod
    def _format_str(_str):
        if _str:
            if _str.find('钢') != -1 or _str.find('混') != -1:
                return '钢混'
            if '-' in _str:
                return None
            if _str.find('附') != -1 or _str.find('属') != -1:
                return '附属'
            if _str.find('养') != -1:
                return '养老用房'
            return _str
